package org.ufba.buscador.crawling.preprocessor.simple;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.ufba.buscador.crawling.preprocessor.interfaces.IStopWordsRemover;
import org.ufba.buscador.util.HTMLUtil;


/**
 *
 * @author Efraim Machado and Arivan Bastos
 */
public class StopWordsRemover implements IStopWordsRemover {

    public static String DEFAULT_STOP_WORDS_FILE_NAME = "org/ufba/buscador/crawling/preprocessor/resources/stop_words.txt";
    public static String DEFAULT_STOP_WORDS_FILE_CHARSET = "UTF-8";
    public static String DEFAULT_STOP_WORD_REPLACEMENT = "";

    private List<String> stopWords;
    //to improve performance
    private String stopWordsRegex;
    
    private String stopWordsFileName = DEFAULT_STOP_WORDS_FILE_NAME;
    private String stopWordsFileCharset = DEFAULT_STOP_WORDS_FILE_CHARSET;
    private String stopWordReplacement = DEFAULT_STOP_WORD_REPLACEMENT;
    
    public List<String> getStopWords() 
    {
		return stopWords;
	}

	public void setStopWords(List<String> stopWords) 
	{
		this.stopWords = stopWords;
		createStopWordsRegex();
	}

	public String getStopWordsFileName() 
	{
		return stopWordsFileName;
	}

	public void setStopWordsFileName(String stopWordsFileName)
	{
		this.stopWordsFileName = stopWordsFileName;
	}

	public String getStopWordsFileCharset() 
	{
		return stopWordsFileCharset;
	}

	public void setStopWordsFileCharset(String stopWordsFileCharset) 
	{
		this.stopWordsFileCharset = stopWordsFileCharset;
	}

	public String getStopWordReplacement() 
	{
		return stopWordReplacement;
	}

	public void setStopWordReplacement(String stopWordReplacement) 
	{
		this.stopWordReplacement = stopWordReplacement;
	}
	
    private void loadStopWords()  
    {
        InputStream fis;
        BufferedReader br;
        String line;
        stopWords = new ArrayList<String>();
        try
        {
        	URL url = ClassLoader.getSystemResource(stopWordsFileName);
        	File file = new File(url.getPath());
            fis = new FileInputStream(file);
            br = new BufferedReader(new InputStreamReader(fis, Charset.forName(stopWordsFileCharset)));
            while ((line = br.readLine()) != null) 
            {
                if (!line.isEmpty())
                {
                    stopWords.add(line.trim());
                }
            }
            createStopWordsRegex();
            // Done with the file
            br.close();
            br = null;
            fis = null;
        } 
        catch (FileNotFoundException ex) 
        {
            Logger.getLogger(StopWordsRemover.class.getName()).log(Level.SEVERE, null, ex);
        }
        catch (IOException ex) 
        {
                Logger.getLogger(StopWordsRemover.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    
    private void createStopWordsRegex() 
    {
    	if (stopWords != null && !stopWords.isEmpty())
    	{
	        StringBuilder stopWordsReg = new StringBuilder();
	        for (int i = 0; i < stopWords.size(); i++) 
	        {
	        	String stopWord = stopWords.get(i); 
	            if (stopWord != null)
	            {
	                stopWordsReg.append("|\\b"+stopWord.trim()+"\\b");
	            }
	        }
	        if (stopWordsReg.length() > 0)
	        {
	        	stopWordsReg.deleteCharAt(0);
	        }
	        stopWordsRegex = "(?i)"+stopWordsReg.toString();
    	}
	}

	public String removeStopWords(String htmlContent) 
	{
        String htmlFinalContent = htmlContent;
        if (stopWords == null || stopWords.isEmpty())
        {
        	loadStopWords();
        }
        htmlFinalContent = removeStopWordsFromString(htmlContent);
        return htmlFinalContent;	
    }
    
    public String removeStopWords(URI uri)
    {
        String htmlFinalContent = HTMLUtil.getHTMLContent(uri);
        return removeStopWordsFromString(htmlFinalContent);
    }

    public String removeStopWords(URL url)
    {
        String htmlFinalContent = HTMLUtil.getHTMLContent(url);
        return removeStopWordsFromString(htmlFinalContent);
    }
    
    private String removeStopWordsFromString(String text) 
    {
        if (text == null)
        {
        	return null;
        }
        
        if (stopWords != null)
        {
        	text = text.replaceAll(stopWordsRegex, stopWordReplacement);
        	text = text.replaceAll("\\s+", " ");
        }
        return text;
    }
}